<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Avoid oversharding | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="how-to.html" title="How To">
<link rel="prev" href="tune-for-disk-usage.html" title="Tune for disk usage">
<link rel="next" href="glossary.html" title="Glossary of terms">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="how-to.html">How To</a></span>
»
<span class="breadcrumb-node">Avoid oversharding</span>
</div>
<div class="navheader">
<span class="prev">
<a href="tune-for-disk-usage.html">« Tune for disk usage</a>
</span>
<span class="next">
<a href="glossary.html">Glossary of terms »</a>
</span>
</div>
<div class="chapter">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="avoid-oversharding"></a>Avoid oversharding<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h2>
</div></div></div>
<p>In some cases, reducing the number of shards in a cluster while maintaining the
same amount of data leads to a more effective use of system resources
(CPU, RAM, IO). In these situations, we consider the cluster <em>oversharded</em>.</p>
<p>The number of shards where this inflection point occurs depends on a variety
of factors, including:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
available hardware
</li>
<li class="listitem">
indexing load
</li>
<li class="listitem">
data volume
</li>
<li class="listitem">
the types of queries executed against the clusters
</li>
<li class="listitem">
the rate of these queries being issued
</li>
<li class="listitem">
the volume of data being queried
</li>
</ul>
</div>
<p>Testing against production data with production queries on production hardware
is the only way to calibrate optimal shard sizes. Shard sizes of tens of GB
are commonly used, and this may be a useful starting point from which to
experiment. Kibana’s <a href="/guide/en/kibana/7.7/elasticsearch-metrics.html" class="ulink" target="_top">Elasticsearch monitoring</a>
provides a useful view of historical cluster performance when evaluating the
impact of different shard sizes.</p>
<h3>
<a id="oversharding-inefficient"></a>Why oversharding is inefficient<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h3>
<p>Each segment has metadata that needs to be kept in heap memory. These include
lists of fields, the number of documents, and terms dictionaries. As a shard
grows in size, the size of its segments generally grow because smaller segments
are <a class="xref" href="index-modules-merge.html" title="Merge">merged</a> into fewer, larger segments. This typically
reduces the amount of heap required by a shard’s segment metadata for a given
data volume. At a bare minimum shards should be at least larger than 1GB to
make the most efficient use of memory.</p>
<p>However, even though shards start to be more memory efficient at around 1GB,
a cluster full of 1GB shards will likely still perform poorly. This is because
having many small shards can also have a negative impact on search and
indexing operations. Each query or indexing operation is executed in a single
thread per shard of indices being queried or indexed to. The node receiving
a request from a client becomes responsible for distributing that request to
the appropriate shards as well as reducing the results from those individual
shards into a single response. Even assuming that a cluster has sufficient
<a class="xref" href="modules-threadpool.html" title="Thread pools">search threadpool threads</a> available to immediately
process the requested action against all shards required by the request, the
overhead associated with making network requests to the nodes holding those
shards and with having to merge the results of results from many small shards
can lead to increased latency. This in turn can lead to exhaustion of the
threadpool and, as a result, decreased throughput.</p>
<h3>
<a id="reduce-shard-counts-increase-shard-size"></a>How to reduce shard counts and increase shard size<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h3>
<p>Try these methods to reduce oversharding.</p>
<h4>
<a id="reduce-shards-for-new-indices"></a>Reduce the number of shards for new indices<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h4>
<p>You can specify the <code class="literal">index.number_of_shards</code> setting  for new indices created
with the <a class="xref" href="indices-create-index.html" title="Create index API">create index API</a> or as part of
<a class="xref" href="indices-templates.html" title="Put index template API">index templates</a> for indices automatically created by
<a class="xref" href="index-lifecycle-management.html" title="ILM: Manage the index lifecycle">index lifecycle management (ILM)</a>.</p>
<p>You can override the <code class="literal">index.number_of_shards</code>  when rolling over an index
using the <a class="xref" href="indices-rollover-index.html#rollover-index-api-example" title="Examples">rollover index API</a>.</p>
<h4>
<a id="create-larger-shards-by-increasing-rollover-thresholds"></a>Create larger shards by increasing rollover thresholds<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h4>
<p>You can roll over indices using the
<a class="xref" href="indices-rollover-index.html" title="Rollover index API">rollover index API</a> or by specifying the
<a class="xref" href="ilm-actions.html#ilm-rollover-action">rollover action</a> in an ILM policy. If using an
ILM policy, increase the rollover condition thresholds (<code class="literal">max_age</code>,
  <code class="literal">max_docs</code>, <code class="literal">max_size</code>)  to allow the indices to grow to a larger size
  before being rolled over, which creates larger shards.</p>
<p>Take special note of any empty indices. These may be managed by an ILM
policy that is rolling over the indices because the <code class="literal">max_age</code> threshold is met.
In this case, you may need to adjust the policy to make use of the <code class="literal">max_docs</code>
or <code class="literal">max_size</code> properties to prevent the creation of these empty indices. One
example where this may happen is if one or more Beats stop sending data. If
the ILM-managed indices for those Beats are configured to roll over
  daily, then new, empty indices will be generated each day. Empty indices can
  be identified using the <a class="xref" href="cat-count.html" title="cat count API">cat count API</a>.</p>
<h4>
<a id="create-larger-shards-with-index-patterns"></a>Create larger shards by using index patterns spanning longer time periods<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h4>
<p>Creating indices covering longer time periods reduces index and shard counts
while increasing index sizes. For example, instead of daily indices, you can
create monthly, or even yearly indices.</p>
<p>If creating indices using Logstash, the
<a href="/guide/en/logstash/7.7/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-index" class="ulink" target="_top">index</a>
property of the Elasticsearch output can be modified to a
<a class="xref" href="date-math-index-names.html" title="Date math support in index names">date math expression</a> covering a longer time period.
For example, use <code class="literal">logstash-%{+YYYY.MM}</code> instead of <code class="literal">logstash-%{+YYYY.MM.dd}</code>
to create monthly, rather than daily, indices. Beats also lets you change the
date math expression defined in the <code class="literal">index</code> property of the Elasticsearch output, such
as for <a href="/guide/en/beats/filebeat/7.7/elasticsearch-output.html#index-option-es" class="ulink" target="_top">Filebeat</a>.</p>
<h4>
<a id="shrink-existing-index-to-fewer-shards"></a>Shrink an existing index to fewer shards<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h4>
<p>You can use the <a class="xref" href="indices-shrink-index.html" title="Shrink index API">shrink index API</a> to shrink an
existing index down to fewer shards.</p>
<p><a class="xref" href="index-lifecycle-management.html" title="ILM: Manage the index lifecycle">index lifecycle management</a> also has a
<a class="xref" href="ilm-actions.html#ilm-shrink-action">shrink action</a> available for indices in the warm phase.</p>
<h4>
<a id="reindex-an-existing-index-to-fewer-shards"></a>Reindex an existing index to fewer shards<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h4>
<p>You can use the <a class="xref" href="docs-reindex.html" title="Reindex API">reindex API</a> to reindex from an existing index
to a new index with fewer shards. After the data has been reindexed, the
oversharded index can be deleted.</p>
<h4>
<a id="reindex-indices-from-shorter-periods-into-longer-periods"></a>Reindex indices from shorter periods into longer periods<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/how-to/avoid-oversharding.asciidoc">edit</a>
</h4>
<p>You can use the <a class="xref" href="docs-reindex.html" title="Reindex API">reindex API</a> to reindex multiple small indices
covering shorter time periods into a larger index covering a longer time period.
For example, daily indices from October with naming patterns such as
<code class="literal">foo-2019.10.11</code> could be combined into a monthly <code class="literal">foo-2019.10</code> index,
like this:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST /_reindex
{
  "source": {
    "index": "foo-2019.10.*"
  },
  "dest": {
    "index": "foo-2019.10"
  }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1330.console"></div>
</div>
<div class="navfooter">
<span class="prev">
<a href="tune-for-disk-usage.html">« Tune for disk usage</a>
</span>
<span class="next">
<a href="glossary.html">Glossary of terms »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
